package com.jinghang.nginxProject.APP

import com.jinghang.nginxProject.utils.StringUtil
import org.apache.spark.sql.SparkSession

/**
  * 解析后的日志格式：
  * ip time url method  protocal http_status bytes_sent referer user_agent
  *
  *
  * ip country province city time method url protocal http_status bytes_sent referer
  * user_agent browser_name browser_version engine_name engine_version os_name platform_name  is_molbie
  */
object _010_LogFormatApp {

  def main(args: Array[String]): Unit = {

    val spark = SparkSession
      .builder()
      .appName("SparkStatFormatJob")
      .master("local[4]")
      .getOrCreate()

    //根据文件路径获取资源
    val path = "data/10000_access.log"
    var sourceRDD = spark.sparkContext.textFile(path, 2)
    //打印输出前十条数据
    sourceRDD.take(10).foreach(println)


    sourceRDD.map(
      //对每一行数据进行处理
      line => {
        val splits = line.split(" ")
        val ip = splits(0)
        /*
          * 原始日志的第三个和第四个字段拼接起来就是完整的访问时间：
          * [10/Nov/2016:00:01:02 +0800] ==> yyyy-MM-dd HH:mm:ss
          */
        val time = splits(3) + " " + splits(4)
        //url去掉 " 单引号
        val url = splits(11).replace("\"", "")

        val method = splits(5).replace("\"", "")
        val protocal = splits(7).replace("\"","")
        val http_status = splits(8)
        val bytes_sent = splits(9)
        val referer = splits(12)
        var user_agent = StringUtil.getUserAgent(line)

        val log_format = ip + "\t" + time + "\t" + url + "\t" + method + "\t" + protocal + "\t" + http_status + "\t" + bytes_sent + "\t" + referer + "\t" + user_agent
        log_format
      }
    )
      .saveAsTextFile("data/output/LogFormatApp1")
    //.take(1000).foreach(println)

    spark.stop()

  }

}
